package crawl.topsy;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.PrintStream;
import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.util.ArrayList;
import java.util.List;

import tools.FileReaderEnhance;
import tools.ObjectTools;

import com.maruti.otterapi.ListParameters;
import com.maruti.otterapi.Otter4JavaException;
import com.maruti.otterapi.TopsyConfig;
import com.maruti.otterapi.authorinfo.AuthorInfo;
import com.maruti.otterapi.authorinfo.AuthorInfoResponse;
import com.maruti.otterapi.experts.Experts;
import com.maruti.otterapi.experts.ExpertsResponse;
import com.maruti.otterapi.linkposts.Linkposts;
import com.maruti.otterapi.linkposts.LinkpostsRequest;
import com.maruti.otterapi.linkposts.LinkpostsResponse;
import com.maruti.otterapi.populartrackbacks.Populartrackbacks;
import com.maruti.otterapi.populartrackbacks.PopulartrackbacksResponse;
import com.maruti.otterapi.search.Post;
import com.maruti.otterapi.search.Search;
import com.maruti.otterapi.search.SearchCount;
import com.maruti.otterapi.search.SearchCriteria;
import com.maruti.otterapi.search.SearchResponse;
import com.maruti.otterapi.searchhistogram.SearchHistogram;
import com.maruti.otterapi.searchhistogram.SearchHistogramResponse;
import com.maruti.otterapi.stats.Stats;
import com.maruti.otterapi.stats.StatsResponse;
import com.maruti.otterapi.top.Top;
import com.maruti.otterapi.top.TopRequest;
import com.maruti.otterapi.top.TopResponse;
import com.maruti.otterapi.trackbacks.TrackbackRequest;
import com.maruti.otterapi.trackbacks.Trackbacks;
import com.maruti.otterapi.trackbacks.TrackbacksResponse;
import com.maruti.otterapi.urlinfo.URLInfo;
import com.maruti.otterapi.urlinfo.URLInfoResponse;

public class TopsyCrawler {

	static String [] API_KEY = FileReaderEnhance.readToString("./data/topsy/key", "UTF-8").split("\n");

	static TopsyCrawler tc = new TopsyCrawler();

	public static void main(String[] args) throws Exception {

		try {
			System.setOut(new PrintStream(new File("./logs/TopsyCrawler.out.log")));
		} catch (FileNotFoundException e) {
			e.printStackTrace();
		}

		System.out.println("Using API Keys from (one key per line) : ./data/topsy/key");
		String queryPath;
		if (args.length > 0) {
			queryPath = args[0];
		} else {
			queryPath = "./data/topsy/query";
		}

		System.out.println("Reading Query from : " + queryPath);
		File queryFile = new File(queryPath);
		String[] queries = FileReaderEnhance.readToString(queryFile, "UTF-8").split("\n");
		for (String line : queries) {
			if (line.startsWith("//")) {
				continue;
			}
			String[] elements = line.split("\t");

			String query = elements[0];
			Long timeStart = Long.parseLong(elements[1]);
			Long timeEnd = Long.parseLong(elements[2]);

			TopsyConfig config = new TopsyConfig();
			config.setApiKeys(API_KEY);
			config.setSetProxy(false);
			System.out.println("Query: " + query);

			// for each query, get the total count first
			int count = tc.searchCount(config, query, timeStart, timeEnd);
			System.out.println("Total hit: " + count + " Time Range: " + timeStart + "," + timeEnd);
			if (count > 0 && count < 1000) {
				System.out.println("\t Hit < 1000, crawling...");
				Search searchTopsy = new Search();
				searchTopsy.setTopsyConfig(config);
				SearchResponse results = null;
				ListParameters parameters = new ListParameters();
				parameters.setAllow_lang("en");
				parameters.setPerpage("100");
				// int pageNum = count % 100 == 0 ? (count / 100) : (count / 100
				// + 1);
				// parameters.setPage(String.valueOf(pageNum));
				try {
					SearchCriteria criteria = new SearchCriteria();
					criteria.setQuery(URLDecoder.decode(query, "UTF-8"));
					criteria.setListParams(parameters);
					criteria.setType("tweet");
					criteria.setNohidden(true);
					results = searchTopsy.search(criteria);
					ObjectTools.writeToFile(results.getResult().getList(),
							"data/objs/topsyCrawl/" + queryFile.getName() + "/" + query.replaceAll(" ", "_"));
				} catch (Otter4JavaException e) {
					e.printStackTrace();
				}
			} else if (count > 1000) {
				List<Post> allPosts = new ArrayList<>();
				// more than 1000, divide the time from the middle
				fetchallPosts(config, query, allPosts, timeStart, timeEnd);
				ObjectTools.writeToFile(allPosts,
						"data/objs/topsyCrawl/" + queryFile.getName() + "/" + query.replaceAll(" ", "_"));
			}
		}
	}

	private static void fetchallPosts(TopsyConfig config, String query, List<Post> allPosts, Long timeStart,
			Long timeEnd) {
		Long timeMiddle = (timeStart + timeEnd) / 2;
		System.out.println("Hit > 1000, Splitting... Time Range: (" + timeStart + "," + timeMiddle + ") and ("
				+ timeMiddle + "," + timeEnd + ")");
		int count1 = tc.searchCount(config, query, timeStart, timeMiddle);
		int count2 = tc.searchCount(config, query, timeMiddle, timeEnd);
		System.out.println("Hit of 1st part: " + count1 + " ; Hit of 2nd Part: " + count2);

		System.out.println("Processing 1st part... Hits: " + count1 + ", From: (" + timeStart + "," + timeMiddle + ")");
		if (count1 > 0 && count1 < 1000) {
			System.out.println("\t Hit < 1000, crawling...");
			Search searchTopsy = new Search();
			searchTopsy.setTopsyConfig(config);
			SearchResponse results = null;
			ListParameters parameters = new ListParameters();
			parameters.setAllow_lang("en");
			parameters.setPerpage("100");
			parameters.setMintime(String.valueOf(timeStart));
			parameters.setMaxtime(String.valueOf(timeMiddle));
			// int pageNum = count1 % 100 == 0 ? (count1 / 100) : (count1 / 100
			// + 1);
			// parameters.setPage(String.valueOf(pageNum));
			try {
				SearchCriteria criteria = new SearchCriteria();
				criteria.setQuery(URLDecoder.decode(query, "UTF-8"));
				criteria.setListParams(parameters);
				criteria.setType("tweet");
				criteria.setNohidden(true);
				results = searchTopsy.search(criteria);
				// put in the list
				allPosts.addAll(results.getResult().getList());
			} catch (Otter4JavaException e) {
				e.printStackTrace();
			} catch (UnsupportedEncodingException e) {
				e.printStackTrace();
			}
		} else if (count1 > 1000) {
			fetchallPosts(config, query, allPosts, timeStart, timeMiddle);
		}

		System.out.println("Processing 2nd part... Hits: " + count2 + ", From: (" + timeMiddle + "," + timeEnd + ")");
		if (count2 > 0 && count2 < 1000) {
			System.out.println("\t Hit < 1000, crawling...");
			Search searchTopsy = new Search();
			searchTopsy.setTopsyConfig(config);
			SearchResponse results = null;
			ListParameters parameters = new ListParameters();
			parameters.setAllow_lang("en");
			parameters.setPerpage("100");
			parameters.setMintime(String.valueOf(timeMiddle));
			parameters.setMaxtime(String.valueOf(timeEnd));
			// int pageNum = count2 % 100 == 0 ? (count2 / 100) : (count2 / 100
			// + 1);
			// parameters.setPage(String.valueOf(pageNum));
			try {
				SearchCriteria criteria = new SearchCriteria();
				criteria.setQuery(URLDecoder.decode(query, "UTF-8"));
				criteria.setListParams(parameters);
				criteria.setType("tweet");
				criteria.setNohidden(true);
				results = searchTopsy.search(criteria);
				// put in the list
				allPosts.addAll(results.getResult().getList());
			} catch (Otter4JavaException e) {
				e.printStackTrace();
			} catch (UnsupportedEncodingException e) {
				e.printStackTrace();
			}
		} else if (count2 > 1000) {
			fetchallPosts(config, query, allPosts, timeMiddle, timeEnd);
		}
	}

	public void search(TopsyConfig config) {
		Search searchTopsy = new Search();
		searchTopsy.setTopsyConfig(config);
		SearchResponse results = null;
		ListParameters parameters = new ListParameters();
		parameters.setAllow_lang("en");
		parameters.setPerpage("100");
		parameters.setPage("1");
		try {
			SearchCriteria criteria = new SearchCriteria();
			criteria.setQuery(URLDecoder.decode("Connecticut Shooting", "UTF-8"));
			criteria.setListParams(parameters);
			criteria.setType("tweet");
			criteria.setNohidden(true);

			results = searchTopsy.search(criteria);
			System.out.println(results.getResult().getList().size());
			System.out.println(results.getResult().getTotal());
			for (Post post : results.getResult().getList()) {
				System.out.println("\t" + post.getContent());
			}
		} catch (Otter4JavaException e) {
			e.printStackTrace();
		} catch (UnsupportedEncodingException e) {
			e.printStackTrace();
		}

		// Use Crawler to retrieve the contents of search results
		// new BasicCrawlController("D:/data/crawler/out",
		// "D:/data/topsy").crawl(results);
	}

	public int searchCount(TopsyConfig config, String query) {
		SearchCount searchCount = new SearchCount();
		searchCount.setTopsyConfig(config);
		SearchResponse response = null;
		try {
			response = searchCount.searchCount(URLDecoder.decode(query, "UTF-8"));
			return Integer.parseInt(response.getResult().getA());
		} catch (Otter4JavaException e) {
			e.printStackTrace();
		} catch (UnsupportedEncodingException e) {
			e.printStackTrace();
		}
		return 0;
	}

	public int searchCount(TopsyConfig config, String query, Long timeStart, Long timeEnd) {
		Search searchTopsy = new Search();
		searchTopsy.setTopsyConfig(config);
		SearchResponse results = null;
		ListParameters parameters = new ListParameters();
		parameters.setAllow_lang("en");
		parameters.setPerpage("10");
		parameters.setPage("1");
		parameters.setMintime(String.valueOf(timeStart));
		parameters.setMaxtime(String.valueOf(timeEnd));
		try {
			SearchCriteria criteria = new SearchCriteria();
			criteria.setQuery(URLDecoder.decode(query, "UTF-8"));
			criteria.setListParams(parameters);
			criteria.setType("tweet");
			criteria.setNohidden(true);
			results = searchTopsy.search(criteria);
			return Integer.parseInt(results.getResult().getTotal());
		} catch (Otter4JavaException | UnsupportedEncodingException e) {
			e.printStackTrace();
		}
		return 0;
	}

	public void searchdate(TopsyConfig config) {
		Search searchTopsy = new Search();
		searchTopsy.setTopsyConfig(config);
		SearchResponse results = null;
		try {
			SearchCriteria criteria = new SearchCriteria();
			criteria.setQuery("barack obama");

			results = searchTopsy.searchdate(criteria);
			System.out.println(results.getResult().getList().size());
			System.out.println(results.getResult().getTotal());
		} catch (Otter4JavaException e) {
			e.printStackTrace();
		}

		// Use Crawler to retrieve the contents of search results
		// new BasicCrawlController("D:/data/crawler/out",
		// "D:/data/topsy").crawl(results);
	}

	public void authorInfo(TopsyConfig config) {
		AuthorInfo authorInfo = new AuthorInfo();
		authorInfo.setTopsyConfig(config);
		AuthorInfoResponse response = null;
		try {
			response = authorInfo.authorInfo("http://twitter.com/barackobama");
			System.out.println(response.getResponse().getName());
			System.out.println(response.getResponse().getInfluence_level());
		} catch (Otter4JavaException e) {
			e.printStackTrace();
		}
	}

	public void experts(TopsyConfig config) {
		Experts experts = new Experts();
		experts.setTopsyConfig(config);
		ExpertsResponse response = null;

		try {
			ListParameters listParams = new ListParameters();
			listParams.setPage("1");

			response = experts.experts("nosql");
			System.out.println(response.getResponse().getList().size());
			System.out.println(response.getResponse().getTotal());
			System.out.println(response.getResponse().getList().get(0).getName());
		} catch (Otter4JavaException e) {
			e.printStackTrace();
		}
	}

	public void searchHistogram(TopsyConfig config) {
		SearchHistogram searchHistogram = new SearchHistogram();
		searchHistogram.setTopsyConfig(config);
		SearchHistogramResponse response = null;

		try {
			response = searchHistogram.searchHistogram("kindle");
		} catch (Otter4JavaException e) {
			e.printStackTrace();
		}
		System.out.println(response.getResponse().getCount_method());
	}

	public void urlInfo(TopsyConfig config) {
		URLInfo urlInfo = new URLInfo();
		urlInfo.setTopsyConfig(config);
		URLInfoResponse response = null;

		try {
			response = urlInfo.urlInfo("http://twitter.com/");
		} catch (Otter4JavaException e) {
			e.printStackTrace();
		}
		System.out.println(response.getResponse().getDescription());
	}

	public void trackbacks(TopsyConfig config) {
		Trackbacks trackbacks = new Trackbacks();
		trackbacks.setTopsyConfig(config);
		TrackbacksResponse results = null;
		try {

			TrackbackRequest request = new TrackbackRequest();
			request.setUrl("http://topsy.com/");

			results = trackbacks.trackbacks(request);
			System.out.println(results.getResponse().getList().size());
			System.out.println(results.getResponse().getTotal());
		} catch (Otter4JavaException e) {
			e.printStackTrace();
		}
	}

	public void top(TopsyConfig config) {
		Top top = new Top();
		top.setTopsyConfig(config);
		TopResponse results = null;
		try {

			TopRequest request = new TopRequest();
			request.setThresh("top100");

			results = top.top(request);
			System.out.println(results.getResponse().getList().size());
			System.out.println(results.getResponse().getTotal());
		} catch (Otter4JavaException e) {
			e.printStackTrace();
		}
	}

	public void stats(TopsyConfig config) {
		Stats stats = new Stats();
		stats.setTopsyConfig(config);
		StatsResponse results = null;
		try {
			results = stats.stats("http://topsy.com");
			System.out.println(results.getResponse().getInfluential());
			System.out.println(results.getResponse().getAll());
		} catch (Otter4JavaException e) {
			e.printStackTrace();
		}
	}

	public void populartrackbacks(TopsyConfig config) {
		Populartrackbacks populartrackbacks = new Populartrackbacks();
		populartrackbacks.setTopsyConfig(config);
		PopulartrackbacksResponse results = null;
		try {
			results = populartrackbacks.populartrackbacks("http://www.google.com/");
			System.out.println(results.getResponse().getTrackback_total());
			System.out.println(results.getResponse().getTopsy_trackback_url());
		} catch (Otter4JavaException e) {
			e.printStackTrace();
		}
	}

	public void linkposts(TopsyConfig config) {
		Linkposts linkposts = new Linkposts();
		linkposts.setTopsyConfig(config);
		LinkpostsResponse results = null;
		try {
			LinkpostsRequest request = new LinkpostsRequest();
			request.setUrl("http://twitter.com/topsy");
			results = linkposts.linkposts(request);
			System.out.println(results.getResponse().getTotal());
			System.out.println(results.getResponse().getList().size());
		} catch (Otter4JavaException e) {
			e.printStackTrace();
		}
	}

	public void linkpostsCount(TopsyConfig config) {
		Linkposts linkposts = new Linkposts();
		linkposts.setTopsyConfig(config);
		LinkpostsResponse results = null;
		try {
			LinkpostsRequest request = new LinkpostsRequest();
			request.setUrl("http://twitter.com/topsy");
			results = linkposts.linkpostsCount(request);
			System.out.println(results.getResponse().getAll());
			System.out.println(results.getResponse().getContains());
		} catch (Otter4JavaException e) {
			e.printStackTrace();
		}
	}
}
